#!/usr/bin/env bash
# make a temp copy of the data, so we can atomically move them into the stream_data input dir
hadoop fs -mkdir temp stream_data
hadoop fs -cp mnist/csv2/train/* temp

# drop data into the stream (monitor spark logs after each command to view behavior)
hadoop fs -mv temp/part-00000 stream_data
hadoop fs -mv temp/part-00001 stream_data
hadoop fs -mv temp/part-0000[2-9] stream_data

# shutdown job, since this normally runs forever, waiting for new data to arrive
# the host and port of the reservation server will be in the driver logs, e.g.
# "listening for reservations at ('gpbl191n01.blue.ygrid.yahoo.com', 38254)"
export PYTHON_ROOT=~/Python
${PYTHON_ROOT}/bin/python ${TFoS_HOME}/tensorflowonspark/reservation_client.py localhost 43509

